{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import sys\n",
    "import time\n",
    "import _init_paths\n",
    "# from core.config import cfg, merge_cfg_from_file, merge_cfg_from_list, assert_and_infer_cfg\n",
    "# from core.test_engine import run_inference, get_inference_dataset\n",
    "# from datasets.json_dataset import JsonDataset\n",
    "import pickle\n",
    "import json\n",
    "from utils.boxes import xywh_to_xyxy, bbox_overlaps\n",
    "import ipdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "hico_pred_path = \"/group/wanbo/HOI_plus/data/cache/hico_train_precomp_boxes_use_coco.json\"\n",
    "hico_gt_path = \"/group/wanbo/HOI_plus/data/hico/hico_annotation.json\"\n",
    "\n",
    "hico_pred_path_test = \"/group/wanbo/HOI_plus/data/cache/hico_train_precomp_boxes_use_coco.json\"\n",
    "hico_pred = json.load(open(hico_pred_path, 'r'))\n",
    "hico_pred_test = json.load(open(hico_pred_path_test, 'r'))\n",
    "hico_gt_path = json.load(open(hico_gt_path, 'r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[221.946044921875, 36.011962890625, 425.5889892578125, 326.634033203125, 0.9810082912445068, 1.0], [67.19271850585938, 92.02972412109375, 574.8057861328125, 399.3364562988281, 0.9993689656257629, 4.0]]\n"
     ]
    }
   ],
   "source": [
    "# print(hico_pred.keys())\n",
    "print(hico_pred['0'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bboxes': [[59, 98, 572, 405], [208, 33, 427, 300]], 'annotations': [{'obj_bbox_idx': 0, 'obj_cate_id': 4, 'action_cate_id': 50, 'action_cate_name': 'race', 'old_action_id': 152, 'obj_cate_name': 'motorcycle', 'hum_bbox_idx': 1}, {'obj_bbox_idx': 0, 'obj_cate_id': 4, 'action_cate_id': 45, 'action_cate_name': 'ride', 'old_action_id': 153, 'obj_cate_name': 'motorcycle', 'hum_bbox_idx': 1}, {'obj_bbox_idx': 0, 'obj_cate_id': 4, 'action_cate_id': 110, 'action_cate_name': 'sit_on', 'old_action_id': 154, 'obj_cate_name': 'motorcycle', 'hum_bbox_idx': 1}, {'obj_bbox_idx': 0, 'obj_cate_id': 4, 'action_cate_id': 100, 'action_cate_name': 'straddle', 'old_action_id': 155, 'obj_cate_name': 'motorcycle', 'hum_bbox_idx': 1}], 'wh': [480, 640], 'filename': 'HICO_train2015_00000001.jpg', 'run_id': 0, 'img_id': 1}\n",
      "{'filename': 'HICO_test2015_00000001.jpg', 'annotations': [{'obj_cate_id': 14, 'obj_bbox_idx': 0, 'action_cate_id': 110, 'action_cate_name': 'sit_on', 'hum_bbox_idx': 1, 'obj_cate_name': 'bench'}, {'obj_cate_id': 14, 'obj_bbox_idx': 0, 'action_cate_id': 110, 'action_cate_name': 'sit_on', 'hum_bbox_idx': 2, 'obj_cate_name': 'bench'}], 'wh': [427, 640], 'bboxes': [[148, 345, 376, 414], [320, 306, 359, 349], [270, 303, 311, 350]], 'run_id': 0, 'img_id': 1}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1.],\n",
       "       [1.],\n",
       "       [1.]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hico_gt_train = hico_gt_path['bbox_train']\n",
    "hico_gt_train[0]\n",
    "hico_gt_test = hico_gt_path['bbox_test']\n",
    "\n",
    "print(hico_gt_train[0])\n",
    "print(hico_gt_test[0])\n",
    "\n",
    "x = np.ones((3,1))\n",
    "x.repeat(1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37633\n",
      "37631\n",
      "total person: 76763 recall person: 57486 recall: 0.7488764118129828\n",
      "total object: 61614 recall object: 35958 recall: 0.5836011296133996\n"
     ]
    }
   ],
   "source": [
    "total_person = 0\n",
    "recall_person = 0\n",
    "\n",
    "total_object = 0\n",
    "recall_object = 0\n",
    "print(len(hico_gt_train))\n",
    "print(len(list(hico_pred.keys())))\n",
    "for iddd, gt_info in enumerate(hico_gt_train):\n",
    "    run_id = gt_info['run_id']\n",
    "    bboxes = gt_info['bboxes']\n",
    "    cates = [0] * len(bboxes)\n",
    "    if len(gt_info['annotations'])!=0:\n",
    "        for anno in gt_info['annotations']:\n",
    "            cates[anno['obj_bbox_idx']] = anno['obj_cate_id']\n",
    "            cates[anno['hum_bbox_idx']] = 1\n",
    "    \n",
    "        ## N1*5, N2*5\n",
    "        gt_idx_bbox = np.concatenate((np.array(bboxes).astype(np.float32), np.array(cates)[:, None].astype(np.float32)), 1)\n",
    "        if hico_pred.get(str(run_id)) is None:\n",
    "            continue\n",
    "        pred_idx_bbox = np.array(hico_pred[str(run_id)]).astype(np.float32)\n",
    "        pred_idx_score = pred_idx_bbox[:,4]\n",
    "        pred_idx_bbox = np.concatenate((pred_idx_bbox[:,:4], pred_idx_bbox[:, -1][:, None]), 1)\n",
    "        pred_idx_score_list = np.argsort(-pred_idx_score) ## sort from large to samll\n",
    "        \n",
    "        num_gt, num_pred = gt_idx_bbox.shape[0], pred_idx_bbox.shape[0]\n",
    "        gt_cate_map = gt_idx_bbox[:,-1][:,None].repeat(num_pred, 1)\n",
    "        pred_cate_map = pred_idx_bbox[:,-1][None, :].repeat(num_gt, 0)\n",
    "        cate_matching = (pred_cate_map==gt_cate_map).astype(np.float32) ## N1*N2\n",
    "        obj_overlap = bbox_overlaps(gt_idx_bbox[:,:4], pred_idx_bbox[:,:4]) ## N1*N2\n",
    "\n",
    "        for score_id in pred_idx_score_list.tolist():\n",
    "            per_obj_overlap = obj_overlap[:, score_id]\n",
    "            per_obj_overlap_binary = (per_obj_overlap>0.5).astype(np.float32)\n",
    "            if per_obj_overlap_binary.sum()>0:\n",
    "#                 print(per_obj_overlap.shape)\n",
    "#                 print(cate_matching)\n",
    "                matching_gt_id = np.argsort(-per_obj_overlap)[0]\n",
    "                if cate_matching[matching_gt_id, score_id]==1:\n",
    "                    if gt_cate_map[matching_gt_id, score_id] == 1: ## here to judge if human box\n",
    "                        recall_person += 1\n",
    "                    else:\n",
    "                        recall_object += 1\n",
    "                obj_overlap[matching_gt_id, :] = 0\n",
    "        \n",
    "        total_person += np.sum(gt_idx_bbox[:, 4]==1)\n",
    "        total_object += (num_gt-np.sum(gt_idx_bbox[:, 4]==1))\n",
    "        \n",
    "        \n",
    "    \n",
    "print('total person:', total_person, 'recall person:', recall_person, 'recall:', recall_person/total_person)\n",
    "print('total object:', total_object, 'recall object:', recall_object, 'recall:', recall_object/total_object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9658\n",
      "9545\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'6919'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-0e823e629f72>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0;31m## N1*5, N2*5\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0mgt_idx_bbox\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbboxes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m         \u001b[0mpred_idx_bbox\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhico_pred_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m         \u001b[0mpred_idx_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpred_idx_bbox\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0mpred_idx_bbox\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred_idx_bbox\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred_idx_bbox\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: '6919'"
     ]
    }
   ],
   "source": [
    "total_person = 0\n",
    "recall_person = 0\n",
    "\n",
    "total_object = 0\n",
    "recall_object = 0\n",
    "print(len(hico_gt_test))\n",
    "print(len(list(hico_pred_test.keys())))\n",
    "for iddd, gt_info in enumerate(hico_gt_test):\n",
    "#     print(iddd)\n",
    "    img_id = gt_info['run_id']\n",
    "\n",
    "    bboxes = gt_info['bboxes']\n",
    "    cates = [0] * len(bboxes)\n",
    "#     if img_id == 6919 or img_id ==9546:\n",
    "#         continue\n",
    "    if len(gt_info['annotations'])!=0:\n",
    "        for anno in gt_info['annotations']:\n",
    "            cates[anno['obj_bbox_idx']] = anno['obj_cate_id']\n",
    "            cates[anno['hum_bbox_idx']] = 1\n",
    "    \n",
    "        ## N1*5, N2*5\n",
    "        gt_idx_bbox = np.concatenate((np.array(bboxes).astype(np.float32), np.array(cates)[:, None].astype(np.float32)), 1)\n",
    "        pred_idx_bbox = np.array(hico_pred_test[str(img_id)]).astype(np.float32)\n",
    "        pred_idx_score = pred_idx_bbox[:,4]\n",
    "        pred_idx_bbox = np.concatenate((pred_idx_bbox[:,:4], pred_idx_bbox[:, -1][:, None]), 1)\n",
    "        pred_idx_score_list = np.argsort(-pred_idx_score) ## sort from large to samll\n",
    "        \n",
    "        num_gt, num_pred = gt_idx_bbox.shape[0], pred_idx_bbox.shape[0]\n",
    "        gt_cate_map = gt_idx_bbox[:,-1][:,None].repeat(num_pred, 1)\n",
    "        pred_cate_map = pred_idx_bbox[:,-1][None, :].repeat(num_gt, 0)\n",
    "        cate_matching = (pred_cate_map==gt_cate_map).astype(np.float32) ## N1*N2\n",
    "        obj_overlap = bbox_overlaps(gt_idx_bbox[:,:4], pred_idx_bbox[:,:4]) ## N1*N2\n",
    "\n",
    "        for score_id in pred_idx_score_list.tolist():\n",
    "            per_obj_overlap = obj_overlap[:, score_id]\n",
    "            per_obj_overlap_binary = (per_obj_overlap>0.5).astype(np.float32)\n",
    "            if per_obj_overlap_binary.sum()>0:\n",
    "#                 print(per_obj_overlap.shape)\n",
    "#                 print(cate_matching)\n",
    "                matching_gt_id = np.argsort(-per_obj_overlap)[0]\n",
    "                if cate_matching[matching_gt_id, score_id]==1:\n",
    "                    if gt_cate_map[matching_gt_id, score_id] == 1: ## here to judge if human box\n",
    "                        recall_person += 1\n",
    "                    else:\n",
    "                        recall_object += 1\n",
    "                obj_overlap[matching_gt_id, :] = 0\n",
    "        \n",
    "        total_person += np.sum(gt_idx_bbox[:, 4]==1)\n",
    "        total_object += (num_gt-np.sum(gt_idx_bbox[:, 4]==1))\n",
    "        \n",
    "        \n",
    "    \n",
    "print('total person:', total_person, 'recall person:', recall_person, 'recall:', recall_person/total_person)\n",
    "print('total object:', total_object, 'recall object:', recall_object, 'recall:', recall_object/total_object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
